# -*- coding: utf-8 -*-
# @Create_time   : :
# @Author        :kumiler
# @emial         :lukunming@jtexpress.com
# @File          :__init__.py
# @Desc          :


from datetime import timedelta
from jms.ods.mysql.complaint_info import jms_ods__complaint_info
from utils.operators.cluster_for_spark_sql_operator import SparkSqlOperator

jms_dwd__dwd_complaint_info_new_base_dt = SparkSqlOperator(
    task_id='jms_dwd__dwd_complaint_info_new_base_dt',
    email= ['lukunming@jtexpress.com','yl_bigdata@yl-scm.com'],
    master='yarn',
    name='jms_dwd__dwd_offline_complaint_new_base_dt_{{ execution_date | date_add(1) | cst_ds }}',
    sql='jms/dwd/sqs/work_order_duplicate/dwd_complaint_info_new_base_dt/execute.sql',
    pool_slots=2,
    executor_cores=3,
    executor_memory='6G',
    num_executors=10,
    conf={'spark.dynamicAllocation.enabled'                  : 'true',
          'spark.shuffle.service.enabled'                    : 'true',
          #'spark.dynamicAllocation.maxExecutors'             : 20,
          'spark.dynamicAllocation.maxExecutors'             : 27,
          'spark.dynamicAllocation.cachedExecutorIdleTimeout': 30,
          'spark.sql.sources.partitionOverwriteMode'         : 'dynamic',
          'spark.executor.memoryOverhead'                    : '2G',
          },
    yarn_queue='pro',
    execution_timeout=timedelta(minutes=30),
)

jms_dwd__dwd_complaint_info_new_base_dt << jms_ods__complaint_info